keras教程-n-循环神经网络的注意力机制的理论和实现

import numpy as np
from keras.models import *
from keras.layers import Input, Dense, merge, Reshape, Flatten, RepeatVector, Permute, Lambda, LSTM
BATCH_SIZE = 12800
TIME_STEP = 10
DEMESION = 10
OUTPUT_LEN = 3
num_sample = 300000
ONE_HOT = False
# 为了结果可重复性
np.random.seed(1337)
def create_data(n):
    '''n: 样本量'''
    
    X = np.zeros((n, TIME_STEP), dtype='int32')
    Y = np.zeros((n,), dtype='int32')
    Y_one_hot = np.zeros((n, OUTPUT_LEN, DEMESION), dtype='int32')
    X_one_hot = np.zeros((n, TIME_STEP, DEMESION), dtype='int32')

    for i in range(n):
        row = np.random.randint(0, 10, (TIME_STEP))
        x1 = row[1]*100+row[2]*10+row[3]
        x2 = row[5]*100+row[6]*10+row[7]
        y = abs(x1-x2)
        y_str = str(y).zfill(0)
        y_digits = [int(s) for s in y_str]
        Y[i]=y
        X[i] = row
        for c, r in enumerate(y_digits):
            Y_one_hot[i, c, r] =1
        for c,r in enumerate(row):
            X_one_hot[i, c, r] =1
            
#     for i in range(10):
#         print(X[i])
#         print(X_one_hot[i])
    return {
        'X': X / 10, 
        'Y':Y / 1000, 
        'X_one_hot':X_one_hot,
        'Y_one_hot':Y_one_hot,
        
    }

def build_dense_model():
    inputs = Input(shape=(TIME_STEP, ))

    # ATTENTION PART STARTS HERE
    # 只对时间步骤加权
    attention_probs = Dense(TIME_STEP, activation='softmax', name='attention_vec')(inputs)
    attention_mul = merge([inputs, attention_probs], output_shape=(TIME_STEP, ), name='attention_mul', mode='mul')
    # ATTENTION PART FINISHES HERE
    print(attention_mul)
    attention_mul = Dense(TIME_STEP, activation='sigmoid')(attention_mul)
    output = Dense(1, activation='sigmoid')(attention_mul)
    print('output:', output)
    model = Model(input=[inputs], output=output)
    print('Input:', inputs)
    print('attention_probs:', attention_probs)
    print('Output:', output)
    return model,inputs, attention_probs

dense_model,inputs, attention_probs = build_dense_model()

dense_model.compile(loss='mean_squared_error',  optimizer='adam',)

data=create_data(num_sample)
dense_model.fit([data['X']], data['Y'], epochs=10)

a_model = Model(inputs, attention_probs)
attention = a_model.predict(data['X'])

%matplotlib inline
import matplotlib.pyplot as plt
plt.bar(range(10), np.mean(attention, axis=0))
print(np.mean(attention, axis=0))

print('真实值',data['Y'][:10])
print('预测值:', dense_model.predict(data['X'][:10]))

def build_lstm_model(only_time_step=True):
    inputs = Input(shape=(TIME_STEP, DEMESION))
    # 转置
    a = Permute((2,1))(inputs)
    # 现在a的形状是: (DEMENSION, TIME_STEP)
    # 目的就是为了得到每个时间步的权重
    a = Dense(TIME_STEP, activation='softmax')(a)
    if only_time_step:
        # 降低一个维度, 使得得到长度为TIME_STEP的向量
        a = Lambda(lambda x: K.mean(x, axis=1), name='dim_reduction')(a)
        weight = a
        # 出重复这个向量得到一个矩阵便于下一步计算
        a = RepeatVector(DEMESION)(a)
        # 现在a的形状是(DEMESION, TIME_STEP)
    # 转置矩阵, 便于下一步计算乘法
    a_probs = Permute((2,1), name='attention_vec')(a)
    # 计算加权后的输入
    weighted_inputs = merge([inputs, a_probs], name='weighted_inputs', mode='mul')
    lstm_units = 64
    # 只需要返回最后的结果
    output = LSTM(lstm_units, return_sequences=False)(weighted_inputs)
    # 经过全连接层
    output = Dense(1, activation='sigmoid')(output)
    model = Model([inputs], output)
    weight_model = Model([inputs], weight)
    return model, weight_model

lstm_model, weight = build_lstm_model()
lstm_model.compile(optimizer='adam', loss='MSE')

lstm_model.fit(data['X_one_hot'], data['Y'], epochs=10)

weightV = weight.predict(data['X_one_hot'])

plt.bar(range(10), np.mean(weightV, axis=0))